import scrapy
# //div[@class="el-pagination is-background"]//button[@class="btn-next"]/@disabled
import os

class A纵横中国网2Spider(scrapy.Spider):
    name = "纵横中国网2"
    async def start(self):

        yield

        for i in range(1,501):
            data={"serialStatus":"1",
                  "vip":"9",
                  "pageNum":f'{i}',
                  "pageSize":"20"}
            yield scrapy.FormRequest('https://www.zongheng.com/api2/catefine/storeSearch',formdata=data)
            # 第几页
            break

    def parse(self, response):
        datas = response.json()['result']['bookList']
        for data in datas:
            bookId = data['bookId']
            name = data['name']
            os.mkdir(f'./static/{name}')
            yield  scrapy.FormRequest('https://bookapi.zongheng.com/api/chapter/getChapterList',formdata={'bookId':f"{bookId}"},callback=self.parse2,meta={'name':name})

            break

    def parse2(self, response):
        # 所有的卷
        ds = response.json()['result']['chapterList']
        ds = ds[1:]
        for data in ds:
            for d in data['chapterViewList']:
                # 章节id
                bookId = d['bookId']
                chapterId = d['chapterId']
                chapterName = d['chapterName']

                link_url =f'https://read.zongheng.com/chapter/{bookId}/{chapterId}.html'
                # print(link_url)
                headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/142.0.0.0 Safari/537.36'}
                yield scrapy.Request(link_url,callback=self.parse3,meta={"chapterName":chapterName,"name":response.meta['name']})


    def parse3(self, response):
        # print(response.text)
        # print(response)
        texts = response.xpath('//div[@class="content"]/p/text()').getall()
        texts = '\n'.join(texts)
        print(texts)
        with open(f'./static/{response.meta['name']}/{response.meta["chapterName"]}.txt','a',encoding='utf-8') as f:
            f.write(texts)




